# -*- coding: utf-8 -*-
'''
Created on 15.05.2012

@author: ronny
'''
import urllib2
import re
import GDL
import numpy
import lxml.html as html
import datetime
import time
import getopt
import sys

startT = int(round(time.time() * 1000))

# constants for dealing with data resources and gdl procedures
DATA_URL = "http://www.stadtentwicklung.berlin.de/umwelt/luftqualitaet/de/messnetz/tageswerte/download/"
GDL_PATH = "/home/ronny/Projekte/Python/GDL_OD/lib/gdl/startup"
GDL_PRO_NAME = "BLUME"
# date constraints for scraping
# there are no data before 2007-01-20
BLUME_MIN_DATE = datetime.date(2007,01,20)
# and no data for today
BLUME_MAX_DATE = datetime.date.today() - datetime.timedelta(days=1)
# scraping / plotting mode
BLUME_MODE_DEFAULT = 1
BLUME_MODE_DAY = 2
BLUME_MODE_PERIOD = 3
# accuracy for period of time
BLUME_ACCURACY_DAY = 1
BLUME_ACCURACY_WEEK = 2
BLUME_ACCURACY_MONTH = 3
BLUME_ACCURACY_YEAR = 4
# xpath queries to scrape the html
# particle names
# 1. row, all elements except first
XQ_PARTICLES = "/html/body/table/tr[2]/td/table/tr/td[2]/table/tr[1]/td[position()>1]//text()[normalize-space()]"
# places
# 1. column, all elements except first, second, last
XQ_SPOTS = "/html/body/table/tr[2]/td/table/tr/td[2]/table/tr[position()>2 and not(position()>last()-1)]/td[1]//text()[normalize-space()]"
# all data from the html table
XQ_DATA = "/html/body/table/tr[2]/td/table/tr/td[2]/table[position()=1]/tr[position()>2 and not(position()>last()-1)]/td[position()>1]//text()[normalize-space()]"
# dictionary for replacing german umlauts from strings
UL_DICT = {u"ö":u"oe", u"ä":u"ae", u"ü":u"ue", u"Ö":u"Oe", u"Ä":u"Ae", u"Ü":u"Ue", u"ß":u"ss"}

# scraping / plotting mode
mode = BLUME_MODE_DAY

# variables for dealing with dates
startDate = -1
endDate = -1
datePeriod = -1

# functions
# print help screen for using this program
def usage():
    print sys.argv[0], "--startDate=20070120, --endDate=20120120, --datePeriod=day"

# replace german umlauts in strings
def replace_ul(string):
    for i, j in UL_DICT.iteritems():
        string = string.replace(i, j)
    return str(string)

# plot data for a specific day containing bar plots of each place
def plot_day(date):
    sys.stdout.write("getting data for: " + str(date) + " ... ")
    
    # open website
    site = urllib2.urlopen(DATA_URL + str(date).replace("-", "") + ".html")
    # parse the website
    doc = html.parse(site)
    # scrape the html using xpath
    # all recorded particles
    particles = doc.xpath(XQ_PARTICLES)
    particles = ";".join(pa.strip("*") for pa in particles if "*" in pa)
    # replace german umlauts because gdl can't plot them
    particles = replace_ul(particles)
    # all recorded places
    spots = doc.xpath(XQ_SPOTS)
    spots = ";".join(sp.lstrip("0123456789 ") for sp in spots if re.search("\d", sp))
    # replace german umlauts because gdl can't plot them
    spots = replace_ul(spots)
    # all recorded data
    data = doc.xpath(XQ_DATA)
    # build numpy array to pass to gdl
    dataArray = numpy.zeros((len(data)))
    for i in range(0, len(data)):
        dat = data[i] if data[i].isdigit() else 0.0
        dataArray[i] = dat
        
    print("done")
    
    # plot
    if startDate == -1:
        GDL.pro(GDL_PRO_NAME, spots, particles, dataArray, BLUME_MODE_DEFAULT)
    else:
        GDL.pro(GDL_PRO_NAME, spots, particles, dataArray, BLUME_MODE_DAY)
    
    print("done")
    
# collect data between two dates and prepare them for plotting
def collectData(start, end, period, dataLength):
    duration = end - start
    dataArray = numpy.zeros(0)
    for i in range(0, duration.days):
        cdate = start + datetime.timedelta(days=i)
        
        sys.stdout.write("getting data for: " + str(cdate) + " ... ")
        
        # open website
        site = urllib2.urlopen(DATA_URL + str(cdate).replace("-", "") + ".html")
        # parse the website
        doc = html.parse(site)
        # all recorded data
        try:
            data = doc.xpath(XQ_DATA)
            tmpArray = numpy.zeros(dataLength)
        
            for i in range(0, len(data)):
                dat = data[i] if data[i].isdigit() else 0.0
                tmpArray[i] = dat
        
            dataArray = numpy.append(dataArray, tmpArray)
        
            print("done")
        except Exception, e:
            print("Error: " + str(e))
        
    return dataArray

# collect data for a period of time and plot the data
def plot_period(startDate, endDate, period):
    print("collecting data for time between: " + str(startDate) + " and " + str(endDate))
    print("Be patient. This will take a while.")
    
    # get particles and spots from the end date
    # open website
    site = urllib2.urlopen(DATA_URL + str(endDate).replace("-", "") + ".html")
    # parse the website
    doc = html.parse(site)
    # scrape the html using xpath
    # all recorded particles
    particles = doc.xpath(XQ_PARTICLES)
    particles = [pa.strip("*") for pa in particles if "*" in pa]
    # all recorded places
    spots = doc.xpath(XQ_SPOTS)
    spots = [sp.lstrip("0123456789 ") for sp in spots if re.search("\d", sp)]
    
    dataLength = len(spots) * (len(particles) * 2)
    
    # all recorded data
    dataArray = collectData(startDate, endDate, period, dataLength)
    # replace german umlauts because gdl can't plot them
    particles = replace_ul((";".join(particles)))
    # replace german umlauts because gdl can't plot them
    spots = replace_ul((";".join(spots)))
    
    GDL.pro(GDL_PRO_NAME, spots, particles, dataArray, BLUME_MODE_PERIOD, str(startDate), str(endDate))

# check for script parameters
# and set variables if necessary
# startDate: startdate for data analysis
# endDate: enddate for data analysis
# datePeriod: accuracy for scraping a period of time (day|week|month|year)
params = "h"
options = ["startDate=", "endDate=", "datePeriod="]
opts = []
args = []
try:
    opts, args = getopt.getopt(sys.argv[1:], params, options)
except getopt.GetoptError:
    print("ERROR: You did not provide all parameters.")
    usage()
    sys.exit()
    
for opt, arg in opts:
    if opt == "-h":
        usage()
    elif opt == "--startDate":
        startDate = datetime.datetime.strptime(arg, "%Y%m%d").date()
    elif opt == "--endDate":
        mode = BLUME_MODE_PERIOD
        endDate = datetime.datetime.strptime(arg, "%Y%m%d").date()
    elif opt == "--datePeriod":
        if arg == "day":
            datePeriod = BLUME_ACCURACY_DAY
        elif arg == "week":
            datePeriod = BLUME_ACCURACY_WEEK
        elif arg == "month":
            datePeriod = BLUME_ACCURACY_MONTH
        elif arg == "year":
            datePeriod = BLUME_ACCURACY_YEAR

# set PY GDL Path to GDL script path
GDL.script(GDL_PATH)

# run data analysis according to script setup
if mode == BLUME_MODE_DAY:
    if startDate == -1:
        plot_day(BLUME_MAX_DATE)
        endT = int(round(time.time() * 1000))
        print "Running time: " + str(endT - startT)
    else:
        plot_day(startDate)
        endT = int(round(time.time() * 1000))
        print "Running time: " + str(endT - startT)
elif mode == BLUME_MODE_PERIOD:
    if startDate == -1:
        print("ERROR: You did not provide a valid start date.")
        usage()
    else:
        plot_period(startDate, endDate, datePeriod)
        endT = int(round(time.time() * 1000))
        print "Running time: " + str(endT - startT)